package com.chance.cc.crawler.development.bootstrap.pdd;

import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.*;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.script.crawler.CrawlerScript.$_type;

/**
 * @author bx
 * @date 2021/4/27 0027 12:51
 */
public class PDDTest1 {

    public static Logger logger = LoggerFactory.getLogger(PDDTest1.class);

    public static void main(String[] args) throws IOException, InterruptedException {
        washData();
//        importGoodIds();
    }

    private static void washData() throws IOException, InterruptedException {
        RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);
        FileOutputStream fenleiOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd_subfenlei.json", true);
        FileOutputStream searchOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd_search.json", true);

        String goodIdsQueue = "pdd-goods-id-queue_set";
        while (true){
            CrawlerRequestRecord requestRecord = redisReader.recordPop("crawler-pdd-classify-queue_list", "crawler-pdd-classify-queue_map");
            if (requestRecord != null){
                if (isUrlMatch(requestRecord.getHttpRequest().getUrl(),"https://\\S*/search\\?\\S*")){
                    search(requestRecord,redisReader,searchOutputStream);
                } else if (isUrlMatch(requestRecord.getHttpRequest().getUrl(),"https://\\S*/api/caterham/query/subfenlei_gyl_label\\?\\S*opt_type=3\\S*")){
                    subfenlei(requestRecord,redisReader,fenleiOutputStream);
                } else if (isUrlMatch(requestRecord.getHttpRequest().getUrl(),"https://\\S*/api/search/mall/plus\\S*")){
                    mail(requestRecord,redisReader);
                }

            } else {
                Thread.sleep(1000);
            }
        }
    }

    private static void importGoodIds() throws IOException, InterruptedException {
        RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);

        BufferedReader bufferedReader = new BufferedReader(new FileReader("D:\\chance\\data\\pdd\\pdd_product.json"));
        String line;
        Set<String> goodsIds = new HashSet<>();
        while ((line = bufferedReader.readLine())!= null){
            Json lineJson = new Json(line);
            String goodsId = lineJson.jsonPath("$.goods.goods_id").get();
            goodsIds.add(goodsId);
        }


        String goodIdsQueue = "pdd-goods-id-queue_set";
        String goodIdsListQueue = "pdd-goods-id-queue_list";
        Set<String> smembers = redisReader.smembers(goodIdsQueue);
        Iterator<String> iterator = smembers.iterator();
        while (iterator.hasNext()){
            String goodsId = iterator.next();
            if (goodsIds.contains(goodsId)){
                iterator.remove();
            }
        }
        for (String smember : smembers) {
            redisReader.listPush(goodIdsListQueue,smember);
        }
    }

    private static void subfenlei(CrawlerRequestRecord requestRecord,RedisReader redisReader,FileOutputStream fileOutputStream){
        HttpPage httpPage = null;
        try {
            String goodIdsQueue = "pdd-fenlei-goods-id-queue_set";
            httpPage = requestRecord.getInternalDownloadPage();
            Json json = new Json(httpPage.getRawText());
            List<String> all = json.jsonPath($_type + ".goods_list").all();
            List<String> results = new ArrayList<>();
            for (String s : all) {
                try {
                    Json goodJson = new Json(s);
                    String goods_id = goodJson.jsonPath("$.goods_id").get();
//                            String goods_id = goodJson.jsonPath("$.item_data.goods_model.goods_id").get();
                    if (redisReader.sismember(goodIdsQueue,goods_id)){
                        System.out.println("重复id："+goods_id);
                    } else {
                        redisReader.sadd(goodIdsQueue,goods_id);
                        results.add(s);
                    }
                } catch (Exception e) {
                    logger.error("wash detail {}",s);
                }
            }
            if (results.size() > 0 ){
                IOUtils.writeLines(results,"\r\n",fileOutputStream,"utf-8");
            }
        } catch (Exception e) {
            logger.error("wash list {}",httpPage.getRawText());
        }
    }

    private static void mail(CrawlerRequestRecord requestRecord,RedisReader redisReader){
        String mallsSetQueue = "pdd-mall-id-queue_set";
        String mallsListQueue = "pdd-mall-id-queue_liat";
        HttpPage httpPage = null;
        try {
            httpPage = requestRecord.getInternalDownloadPage();
            Json json = new Json(httpPage.getRawText());
            List<String> all = json.jsonPath($_type + ".items").all();
            for (String s : all) {
                try {
                    Json mallJson = new Json(s);
                    String mall_id = mallJson.jsonPath("$.mall_id").get();
                    if (redisReader.sismember(mallsSetQueue,mall_id)){
                        System.out.println("重复mall id："+mall_id);
                    } else {
                        redisReader.sadd(mallsSetQueue,mall_id);
                        redisReader.listPush(mallsListQueue,mall_id);
                    }
                } catch (Exception e) {
                    logger.error("wash detail {}",s);
                }
            }
//            IOUtils.writeLines(all,"\r\n",fileOutputStream,"utf-8");
        } catch (Exception e) {
            logger.error("wash list {}",httpPage.getRawText());
        }
    }

    private static void search(CrawlerRequestRecord requestRecord,RedisReader redisReader,FileOutputStream fileOutputStream){
        String goodIdsQueue = "pdd-search-goods-id-queue_set";
        logger.info("searc url {}",requestRecord.getHttpRequest().getUrl());
        HttpPage httpPage = null;
        try {

            List<String> results = new ArrayList<>();
            httpPage = requestRecord.getInternalDownloadPage();
            Json json = new Json(httpPage.getRawText());
            List<String> all = json.jsonPath($_type + ".items").all();
            for (String s : all) {
                try {
                    Json goodJson = new Json(s);
                    String goods_id = goodJson.jsonPath("$.item_data.goods_model.goods_id").get();
                    if (redisReader.sismember(goodIdsQueue,goods_id)){
                        System.out.println("重复id："+goods_id);
                    }else {
                        redisReader.sadd(goodIdsQueue,goods_id);
                        results.add(s);
                    }
                } catch (Exception e) {
                    logger.error("wash detail {}",s);
                }
            }
            if (results.size() > 0 ){
                IOUtils.writeLines(results,"\r\n",fileOutputStream,"utf-8");
            }
        } catch (Exception e) {
            logger.error("wash list {}",httpPage.getRawText());
        }
    }

    public static boolean isUrlMatch(String requestUrl, String urlRegx){
        Pattern pattern=Pattern.compile(urlRegx);
        return pattern.matcher(requestUrl).matches();
    }

}
